





<!DOCTYPE html>
<html class="writer-html5" lang="zh-CN" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Design and Architecture &mdash; tvm 0.8.dev1982 文档</title>
  

  
  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tlcpack_theme.css" type="text/css" />

  
  
    <link rel="shortcut icon" href="../_static/tvm-logo-square.png"/>
  

  
  
  
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/translations.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
    <script type="text/javascript" src="../_static/js/tlcpack_theme.js"></script>
    <link rel="index" title="索引" href="../genindex.html" />
    <link rel="search" title="搜索" href="../search.html" />
    <link rel="next" title="TVM runtime系统" href="runtime.html" />
    <link rel="prev" title="Python Target Parametrization" href="../dev/how_to/pytest_target_parametrization.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    
<header class="header">
    <div class="innercontainer">
      <div class="headerInner d-flex justify-content-between align-items-center">
          <div class="headerLogo">
               <a href="https://tvm.apache.org/"><img src=https://tvm.apache.org/assets/images/logo.svg alt="logo"></a>
          </div>

          <div id="headMenu" class="headerNav">
            <button type="button" id="closeHeadMenu" class="navCloseBtn"><img src="../_static/img/close-icon.svg" alt="Close"></button>
             <ul class="nav">
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/community>Community</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/download>Download</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/vta>VTA</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/blog>Blog</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/docs>Docs</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmconf.org>Conference</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://github.com/apache/tvm/>Github</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmchinese.github.io/declaration_zh_CN.html>About-Translators</a>
                </li>
             </ul>
               <div class="responsivetlcdropdown">
                 <button type="button" class="btn-link">
                   ASF
                 </button>
                 <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                 </ul>
               </div>
          </div>
            <div class="responsiveMenuIcon">
              <button type="button" id="menuBtn" class="btn-menu"><img src="../_static/img/menu-icon.svg" alt="Menu Icon"></button>
            </div>

            <div class="tlcDropdown">
              <div class="dropdown">
                <button type="button" class="btn-link dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
                  ASF
                </button>
                <div class="dropdown-menu dropdown-menu-right">
                  <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                  </ul>
                </div>
              </div>
          </div>
       </div>
    </div>
 </header>
 
    <nav data-toggle="wy-nav-shift" class="wy-nav-side fixed">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html">
          

          
            
            <img src="../_static/tvm-logo-small.png" class="logo" alt="Logo"/>
          
          </a>

          
            
            
                <div class="version">
                  0.8.dev1982
                </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption" role="heading"><span class="caption-text">如何开始</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../install/index.html">安装 TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contribute/index.html">贡献者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">用户引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tutorial/index.html">User Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../how_to/index.html">How To Guides</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">开发者引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../dev/tutorial/index.html">Developer Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dev/how_to/how_to.html">开发者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">架构指南</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="current reference internal" href="#">Design and Architecture</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#example-compilation-flow">编译流程示例</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#key-data-structures">关键数据结构</a></li>
<li class="toctree-l3"><a class="reference internal" href="#transformations">转换</a></li>
<li class="toctree-l3"><a class="reference internal" href="#search-space-and-learning-based-transformations">搜索空间和基于学习的转换</a></li>
<li class="toctree-l3"><a class="reference internal" href="#target-translation">目标转化</a></li>
<li class="toctree-l3"><a class="reference internal" href="#runtime-execution">Runtime 执行</a></li>
<li class="toctree-l3"><a class="reference internal" href="#summary-and-discussions">总结与讨论</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#logical-architecture-components">逻辑架构组件</a></li>
<li class="toctree-l2"><a class="reference internal" href="#tvm-support">tvm/support</a></li>
<li class="toctree-l2"><a class="reference internal" href="#tvm-runtime">tvm/runtime</a><ul>
<li class="toctree-l3"><a class="reference internal" href="runtime.html">TVM runtime系统</a></li>
<li class="toctree-l3"><a class="reference internal" href="runtime.html#runtime-specific-information">Runtime-Specific Information</a></li>
<li class="toctree-l3"><a class="reference internal" href="debugger.html">Debugger</a></li>
<li class="toctree-l3"><a class="reference internal" href="virtual_machine.html">Putting the VM in TVM: The Relay Virtual Machine</a></li>
<li class="toctree-l3"><a class="reference internal" href="introduction_to_module_serialization.html">Introduction to Module Serialization</a></li>
<li class="toctree-l3"><a class="reference internal" href="device_target_interactions.html">Device/Target Interactions</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#tvm-node">tvm/node</a></li>
<li class="toctree-l2"><a class="reference internal" href="#tvm-ir">tvm/ir</a><ul>
<li class="toctree-l3"><a class="reference internal" href="pass_infra.html">Pass Infrastructure</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#tvm-target">tvm/target</a><ul>
<li class="toctree-l3"><a class="reference internal" href="device_target_interactions.html">Device/Target Interactions</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#tvm-tir">tvm/tir</a></li>
<li class="toctree-l2"><a class="reference internal" href="#tvm-arith">tvm/arith</a></li>
<li class="toctree-l2"><a class="reference internal" href="#tvm-te">tvm/te</a><ul>
<li class="toctree-l3"><a class="reference internal" href="inferbound.html">InferBound Pass</a></li>
<li class="toctree-l3"><a class="reference internal" href="hybrid_script.html">混合前端开发指南</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#tvm-topi">tvm/topi</a></li>
<li class="toctree-l2"><a class="reference internal" href="#tvm-relay">tvm/relay</a><ul>
<li class="toctree-l3"><a class="reference internal" href="relay_intro.html">Relay 介绍</a></li>
<li class="toctree-l3"><a class="reference internal" href="relay_op_strategy.html">Relay Operator Strategy</a></li>
<li class="toctree-l3"><a class="reference internal" href="convert_layout.html">Convert Layout Pass</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#tvm-autotvm">tvm/autotvm</a><ul>
<li class="toctree-l3"><a class="reference internal" href="benchmark.html">Benchmark Performance Log Format</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#frontends">Frontends</a><ul>
<li class="toctree-l3"><a class="reference internal" href="frontend/tensorflow.html">TensorFlow 前端</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#security">安全</a><ul>
<li class="toctree-l3"><a class="reference internal" href="security.html">Security Guide</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#microtvm">microTVM</a><ul>
<li class="toctree-l3"><a class="reference internal" href="microtvm_design.html">microTVM设计文档</a></li>
<li class="toctree-l3"><a class="reference internal" href="model_library_format.html">Model Library Format</a></li>
</ul>
</li>
</ul>
</li>
</ul>
<p class="caption" role="heading"><span class="caption-text">主题引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../topic/microtvm/index.html">microTVM：裸机使用TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../topic/vta/index.html">VTA: Versatile Tensor Accelerator</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">参考指南</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../reference/langref/index.html">语言参考</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/api/python/index.html">Python API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/api/links.html">Other APIs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/publications.html">Publications</a></li>
<li class="toctree-l1"><a class="reference internal" href="../genindex.html">索引</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
      
      <nav class="wy-nav-top" aria-label="top navigation" data-toggle="wy-nav-top">
        
            <div class="togglemenu">

            </div>
            <div class="nav-content">
              <!-- tvm -->
              Table of content
            </div>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        

          




















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> <span class="br-arrow">></span></li>
        
      <li>Design and Architecture</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../_sources/arch/index.rst.txt" rel="nofollow"> <img src="../_static//img/source.svg" alt="viewsource"/></a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="design-and-architecture">
<h1>Design and Architecture<a class="headerlink" href="#design-and-architecture" title="永久链接至标题">¶</a></h1>
<p>本文档适用于想要了解TVM架构和/或积极开发项目的开发人员。本页的布局如下：</p>
<ul class="simple">
<li><p><a href="#id1"><span class="problematic" id="id2">`</span></a>编译流程示例`_概述了TVM所采取的将一个模型的高层描述转化为一个可部署的模块的步骤。 请从阅读本节开始。</p></li>
<li><p><a class="reference internal" href="#logical-architecture-components">逻辑体系架构组件</a> 章节 描述了逻辑组件。后面的部分按组件的名称，分为每个部件的特定指南。</p></li>
<li><p>The <a class="reference internal" href="device_target_interactions.html#tvm-target-specific-overview"><span class="std std-ref">Device/Target Interactions</span></a>
page describes how TVM interacts with each supported physical device
and code-generation target.</p></li>
<li><p>请随意查看 :ref:<a href="#id1"><span class="problematic" id="id2">`</span></a>开发者指南`以寻求有用的开发技巧</p></li>
</ul>
<p>这个指南补充了一些架构的补充视图。首先我们回顾一个单个端到端的编译流程，并讨论关键的数据结构和转换。这个基于runtime的视图，主要关注运行编译器时，各个组件的交互。然后我们会回顾各逻辑模块的基本代码和他们之间的关系。这一部分提供了一个设计的静态的总体视图。</p>
<div class="section" id="example-compilation-flow">
<h2>编译流程示例<a class="headerlink" href="#example-compilation-flow" title="永久链接至标题">¶</a></h2>
<p>本教程中，我们会学习在编译器中的一个编译流程示例。下图展示了这个流程。在高层次上，包含了几个步骤。</p>
<ul class="simple">
<li><p>导入：前端组件将模型摄入变成一个IR模块，这个IR模块包含了一批函数，这些函数在内部表示了这个模型。</p></li>
<li><p>转换：编译器将一个IR模块转换为另一个在功能上等效或大约等效（例如：在量化情况下）的IR模块。许多转换是独立于目标（即后端）的。我们还允许目标来影响转换pipeline的配置。</p></li>
<li><p>目标转化：编译器将IR模块转化为（codegen）一个可以被目标指定的可执行格式。这个目标转化的结果会被封装成可以在目标runtime环境下被输出，读取和执行的 <cite>runtime.Module</cite> 。</p></li>
<li><p>Runtime执行：用户读回一个 <a href="#id1"><span class="problematic" id="id2">`</span></a>runtime.Module`并且在支持的runtime环境下运行编译后的函数。</p></li>
</ul>
<div class="figure align-center">
<a class="reference internal image-reference" href="https://raw.githubusercontent.com/tlc-pack/web-data/main/images/design/tvm_dyn_workflow.svg"><img alt="https://raw.githubusercontent.com/tlc-pack/web-data/main/images/design/tvm_dyn_workflow.svg" src="https://raw.githubusercontent.com/tlc-pack/web-data/main/images/design/tvm_dyn_workflow.svg" width="85%" /></a>
</div>
<div class="section" id="key-data-structures">
<h3>关键数据结构<a class="headerlink" href="#key-data-structures" title="永久链接至标题">¶</a></h3>
<p>设计和理解一个复杂系统的最佳方法之一就是去明确关键的数据结构和操纵（转换）这些数据结构的api。一旦我们明确了关键的数据结构，我们就可以将系统拆分为逻辑组件，这些逻辑组件可以定义一批关键数据结构或者定义数据结构之间的转换。</p>
<p><a href="#id1"><span class="problematic" id="id2">**</span></a>IRModule**是整个堆栈中主要的数据结构。一个IR模块（IRmodule，中间表示模块）包含一组函数。目前我们支持两种主要的函数变体。</p>
<ul class="simple">
<li><p><strong>relay::Function</strong> 是一种高级函数程序表示。一个relay.Function通常对应一个端到端模型。你可以将relay.Function看作一个计算图，额外带有对控制流，递归和复杂数据结构的支持。</p></li>
<li><p><strong>tir::PrimFunc</strong> 是一种低级程序表示，包含元素，包含循环嵌套选择、多维加载/存储、线程和向量/张量指令等元素。它通常被用于表示在一个模型模型中处理一个（可能融合的）层的算子程序。</p></li>
</ul>
<p>在编译时，一个relay 函数可能被底层化为多个tir::PrimFunc 函数 和一个调用这些tir::PrimFunc函数的顶层函数。</p>
</div>
<div class="section" id="transformations">
<h3>转换<a class="headerlink" href="#transformations" title="永久链接至标题">¶</a></h3>
<p>现在我们已经介绍了关键数据结构，让我们来讨论转换。每个转换都被可用于以下目的之一：</p>
<ul class="simple">
<li><p>优化：将程序转换为一个等效的或者更优化的版本。</p></li>
<li><p>底层化：将一个程序转化为更靠近目标，更底层的表达式。</p></li>
</ul>
<p><a href="#id1"><span class="problematic" id="id2">**</span></a>relay/transform**包含一组优化模型的passes。优化包含常见的程序优化比如常数折叠和死码消除，以及张量特定的passes例如布局转换和比例因子折叠。</p>
<p>在relay优化pipeline的末尾，我们会运行一个pass（FuseOps）来将端到端函数（例如：MobileNet）拆分为子函数（例如：conv2d-relu）段。我们把这些称为函数段。这个过程帮助我们将原始的问题分为两个子问题。</p>
<ul class="simple">
<li><p>子函数的编译和优化。</p></li>
<li><p>总体执行结构：我们需要对生成的子函数做一系列的调用来执行整个模型。</p></li>
</ul>
<p>我们用底层tir阶段来编译和优化每个子函数。对特定的目标，我们也可以直接进入目标转化阶段并使用外部代码生成器。</p>
<p>有几种不同的方法（在relay/backend）里来处理对整个执行问题的调用。对于具有已知形状且没有控制流的简单模型，我们可以底层化为一个将执行结构存储在图像里的图像执行器。我们也支持用于动态执行的虚拟机后端。最后，我们计划支持提前编译，将高层执行结构编译为可执行的和被生成的原语函数。所有的这些执行模式都被封装在一个同意的 <strong>runtime.Module</strong> 接口，我们将在这个指南的后半部分讨论这个接口。</p>
<p><a href="#id1"><span class="problematic" id="id2">**</span></a>tir/transform**包含TIR层函数的转换passes。很多tir passes的目的是底层化。例如，有些passes将多维访问扁平化为一维指针访问，来内部函数扩展为目标指定函数，以及修饰函数入口来满足runtime调用习惯。当然，也有一些优化passes比如访问索引简化和死码消除。</p>
<p>LLVM, CUDA C 可以和其他目标编译器可以在目标阶段进行许多底层优化。所以，我们将底层优化如寄存器分配留给下游编译器，而只关注于它们未能涵盖的优化。</p>
</div>
<div class="section" id="search-space-and-learning-based-transformations">
<h3>搜索空间和基于学习的转换<a class="headerlink" href="#search-space-and-learning-based-transformations" title="永久链接至标题">¶</a></h3>
<p>目前为止，我们讨论过的转换过程都是确定和基于规则的。TVM堆栈的一个设计目标就是去支持不同硬件平台上的高性能代码优化。为此，我们需要尽可能多的研究优化的选择，包括但不限于多维张量访问，循环平铺表现，特殊的加速器内存层级和线程。</p>
<p>很难定义一种启发式方法来决定所有的选择。相反，我们将采取基于搜索和学习的方法。我们首先定义一组可以用来转换程序的操作。示例操作包括循环转换、内联、向量化。我们将这些操作称为**调度原语**。调度原语的集合定义了我们可以对程序进行优化的搜索空间。然后，系统搜索不同的可能调度序列来选择最佳调度组合。搜索过程通常由机器学习算法引导。</p>
<p>一旦搜索完成，我们可以为（可能融合的）算子记录最佳调度序列。然后，编译器可以查找最佳调度序列并将其应用于程序。值得注意的是，此调度应用程序阶段与基于规则的转换**完全相同**，使我们能够与传统的passes共享接口协定。</p>
<p>我们使用基于搜索的优化来处理初始tir函数生成问题。该模块的这一部分称为AutoTVM（自动调度程序）。随着我们继续开发TVM堆栈，我们希望将基于学习的转换扩展到更多领域。</p>
</div>
<div class="section" id="target-translation">
<h3>目标转化<a class="headerlink" href="#target-translation" title="永久链接至标题">¶</a></h3>
<p>目标转换阶段将IRModule转换为对应目标可执行的文件格式。对于x86和ARM等后端，我们使用LLVM IRBuilder构建内存中的LLVM IR。我们还可以生成源代码级语言，如CUDA C和OpenCL。最后，我们支持通过外部代码生成器将Relay函数（子图）直接转化到特定目标。重要的是，最终代码生成阶段应尽可能轻简。绝大多数转换和底层化应在目标转化阶段之前执行。</p>
<p>我们还提供了一个目标结构来特定编译目标。目标转化阶段之前的变换也会受到目标的影响-例如，目标的向量长度会改变向量化行为。</p>
</div>
<div class="section" id="runtime-execution">
<h3>Runtime 执行<a class="headerlink" href="#runtime-execution" title="永久链接至标题">¶</a></h3>
<p>TVM runtime的主要目的是提供一个最小的API用于加载和执行编译后的文件，使用它们选择的语言包括Python，C++，Rust， Go，Java 还有JavaScript。以下的代码片段展示了一个使用Python的例子。</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tvm</span>
<span class="c1"># Example runtime execution program in python, with type annotated</span>
<span class="n">mod</span><span class="p">:</span> <span class="n">tvm</span><span class="o">.</span><span class="n">runtime</span><span class="o">.</span><span class="n">Module</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">runtime</span><span class="o">.</span><span class="n">load_module</span><span class="p">(</span><span class="s2">&quot;compiled_artifact.so&quot;</span><span class="p">)</span>
<span class="n">arr</span><span class="p">:</span> <span class="n">tvm</span><span class="o">.</span><span class="n">runtime</span><span class="o">.</span><span class="n">NDArray</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">nd</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">tvm</span><span class="o">.</span><span class="n">cuda</span><span class="p">(</span><span class="mi">0</span><span class="p">))</span>
<span class="n">fun</span><span class="p">:</span> <span class="n">tvm</span><span class="o">.</span><span class="n">runtime</span><span class="o">.</span><span class="n">PackedFunc</span> <span class="o">=</span> <span class="n">mod</span><span class="p">[</span><span class="s2">&quot;addone&quot;</span><span class="p">]</span>
<span class="n">fun</span><span class="p">(</span><span class="n">a</span><span class="p">)</span>
<span class="k">print</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
</pre></div>
</div>
<p><a class="reference internal" href="../reference/api/python/runtime.html#tvm.runtime.Module" title="tvm.runtime.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">tvm.runtime.Module</span></code></a> 封装了编译的结果。一个runtime.Module 包含一个Getfunction 方法来按名称获得PackedFuncs。</p>
<p>:py:class:<a href="#id1"><span class="problematic" id="id2">`</span></a>tvm.runtime.PackedFunc`是两种生成函数的类型擦除函数接口。一个 runtime.PackedFunc 可以接受参数然后返回以下类型的返回值: POD 类型(int, float), string, runtime.PackedFunc, runtime.Module, runtime.NDArray, 和其他runtime.Object的子类。</p>
<p><a class="reference internal" href="../reference/api/python/runtime.html#tvm.runtime.Module" title="tvm.runtime.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">tvm.runtime.Module</span></code></a> 和 <a class="reference internal" href="../reference/api/python/runtime.html#tvm.runtime.PackedFunc" title="tvm.runtime.PackedFunc"><code class="xref py py-class docutils literal notranslate"><span class="pre">tvm.runtime.PackedFunc</span></code></a> 是模块化runtime的强大机制。 例如, 为了在CUDA得到上述 <cite>addone</cite> 函数,我们可以使用 LLVM 来生成主机端代码以计算启动参数(如线程组的大小) 然后从CUDA驱动程序API支持的CUDAModule调用另一个PackedFunc。同样的机制也可以用于OpenCL内核。</p>
<p>上述例子仅仅设计了一个简单的 <cite>addone</cite> 函数。下列的代码片段给了一个端到端模型使用相同接口来执行的例子。</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tvm</span>
<span class="c1"># Example runtime execution program in python, with types annotated</span>
<span class="n">factory</span><span class="p">:</span> <span class="n">tvm</span><span class="o">.</span><span class="n">runtime</span><span class="o">.</span><span class="n">Module</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">runtime</span><span class="o">.</span><span class="n">load_module</span><span class="p">(</span><span class="s2">&quot;resnet18.so&quot;</span><span class="p">)</span>
<span class="c1"># Create a stateful graph execution module for resnet18 on cuda(0)</span>
<span class="n">gmod</span><span class="p">:</span> <span class="n">tvm</span><span class="o">.</span><span class="n">runtime</span><span class="o">.</span><span class="n">Module</span> <span class="o">=</span> <span class="n">factory</span><span class="p">[</span><span class="s2">&quot;resnet18&quot;</span><span class="p">](</span><span class="n">tvm</span><span class="o">.</span><span class="n">cuda</span><span class="p">(</span><span class="mi">0</span><span class="p">))</span>
<span class="n">data</span><span class="p">:</span> <span class="n">tvm</span><span class="o">.</span><span class="n">runtime</span><span class="o">.</span><span class="n">NDArray</span> <span class="o">=</span> <span class="n">get_input_data</span><span class="p">()</span>
<span class="c1"># set input</span>
<span class="n">gmod</span><span class="p">[</span><span class="s2">&quot;set_input&quot;</span><span class="p">](</span><span class="mi">0</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span>
<span class="c1"># execute the model</span>
<span class="n">gmod</span><span class="p">[</span><span class="s2">&quot;run&quot;</span><span class="p">]()</span>
<span class="c1"># get the output</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">gmod</span><span class="p">[</span><span class="s2">&quot;get_output&quot;</span><span class="p">](</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span>
</pre></div>
</div>
<p>主要结论是，runtime.Module 和 runtime.PackedFunc足够处理算子级别的程序（比如addone）以及端到端模型。</p>
</div>
<div class="section" id="summary-and-discussions">
<h3>总结与讨论<a class="headerlink" href="#summary-and-discussions" title="永久链接至标题">¶</a></h3>
<p>总的来说，编译流中的关键数据结构是：</p>
<ul class="simple">
<li><p>IRModule: 包含 relay.Function 和 tir.PrimFunc。</p></li>
<li><p>runtime.Module: 包含 runtime.PackedFunc。</p></li>
</ul>
<p>编译的大部分流程是关键数据结构之间的转换。</p>
<ul class="simple">
<li><p>relay/transform 和 tir/transform 是基于规则的确定性转换。</p></li>
<li><p>auto_scheduler 和 autotvm 包含基于搜索的转换。</p></li>
</ul>
<p>最后，编译流程示例只是TVM堆栈的一个经典用例。我们将这些关键数据结构和转换提供pyhon和C++API。因此，你可以使用TVM就像使用numpy一样，除了关注的数据结构从numpy.ndarray 变成 tvm.IRModule。一下是一些使用示例。</p>
<ul class="simple">
<li><p>用python API直接构造 IRModule。</p></li>
<li><p>组成一组自定变换（如自定义量化）。</p></li>
<li><p>直接使用TVM提供的python API操作IR。</p></li>
</ul>
</div>
</div>
<div class="section" id="logical-architecture-components">
<h2>逻辑架构组件<a class="headerlink" href="#logical-architecture-components" title="永久链接至标题">¶</a></h2>
<div class="figure align-center" id="id1">
<a class="reference internal image-reference" href="https://raw.githubusercontent.com/tlc-pack/web-data/main/images/design/tvm_static_overview.svg"><img alt="https://raw.githubusercontent.com/tlc-pack/web-data/main/images/design/tvm_static_overview.svg" src="https://raw.githubusercontent.com/tlc-pack/web-data/main/images/design/tvm_static_overview.svg" width="85%" /></a>
<p class="caption"><span class="caption-text">TVM体系架构图</span><a class="headerlink" href="#id1" title="永久链接至图片">¶</a></p>
</div>
<p>上方图片展示了项目中的主要逻辑组件。有关组件及其之间关系的信息请阅读接下来的章节。</p>
</div>
<div class="section" id="tvm-support">
<h2>tvm/support<a class="headerlink" href="#tvm-support" title="永久链接至标题">¶</a></h2>
<p>support 模块包含基础结构最常用的实用程序，例如通用arena分配器，套接字和日志。</p>
</div>
<div class="section" id="tvm-runtime">
<h2>tvm/runtime<a class="headerlink" href="#tvm-runtime" title="永久链接至标题">¶</a></h2>
<p>runtime 是 TVM 堆栈的基础。它提供了加载和执行编译文件的机制。runtime给前端语言如 Python 和 Rust，定义了一套稳定的 C API 接口。</p>
<p><cite>runtime::Object</cite> 是除了 <a href="#id1"><span class="problematic" id="id2">`</span></a>runtime::PackedFunc`之外在TVM runtime里面一个重要的数据结构.它是一个引用计数基类，具有类型索引，给runtime类型检查和向下转型提供支持。Object 系统允许开发者向runtime引进新的数据结构，比如数组，映射还有新的IR数据结构。</p>
<p>除了部署应用之外，编译器本身还大量使用TVM runtime机制。所有的IR数据结构都是`runtime::Object`的子类，所以它们能直接被python前端直接访问和操作。我们用PackedFunc 机制向前端公开各种API。</p>
<p>支持不同硬件后端的runtime在runtime子目录里定义（如runtime/opencl）。这些硬件特定的runtime模块定义了设备内存分配和设备函数的序列化的API。</p>
<p><a href="#id1"><span class="problematic" id="id2">`</span></a>runtime/rpc`实现对PackedFunc的rpc支持。我们可以使用RPC机制将交叉编译库发送到远程设备，并对执行性能进行基准测试。rpc基础设施支持从各种硬件后端收集数据，以进行基于学习的优化。</p>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="runtime.html">TVM runtime系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="runtime.html#runtime-specific-information">Runtime-Specific Information</a></li>
</ul>
</div>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="debugger.html">Debugger</a></li>
<li class="toctree-l1"><a class="reference internal" href="virtual_machine.html">Putting the VM in TVM: The Relay Virtual Machine</a></li>
<li class="toctree-l1"><a class="reference internal" href="introduction_to_module_serialization.html">Introduction to Module Serialization</a></li>
<li class="toctree-l1"><a class="reference internal" href="device_target_interactions.html">Device/Target Interactions</a></li>
</ul>
</div>
</div>
<div class="section" id="tvm-node">
<h2>tvm/node<a class="headerlink" href="#tvm-node" title="永久链接至标题">¶</a></h2>
<p>node模块给IR数据结构在 <a href="#id1"><span class="problematic" id="id2">`</span></a>runtime::Object`之上添加了其他功能。主要功能包括反射、序列化、结构等价和散列。</p>
<p>多亏了node模块，我们可以使用TVM IRNode在python里面的名字来访问它们的任何字段。</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">x</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">tir</span><span class="o">.</span><span class="n">Var</span><span class="p">(</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="s2">&quot;int32&quot;</span><span class="p">)</span>
<span class="n">y</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">tir</span><span class="o">.</span><span class="n">Add</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
<span class="c1"># a and b are fields of a tir.Add node</span>
<span class="c1"># we can directly use the field name to access the IR structures</span>
<span class="k">assert</span> <span class="n">y</span><span class="o">.</span><span class="n">a</span> <span class="o">==</span> <span class="n">x</span>
</pre></div>
</div>
<p>我们还可以将任意IR node序列化为JSON格式，然后重新加载它们。保存/存储和检查IRnode的能力为编译器更好的可访问性提供了基础。</p>
</div>
<div class="section" id="tvm-ir">
<h2>tvm/ir<a class="headerlink" href="#tvm-ir" title="永久链接至标题">¶</a></h2>
<p>The <cite>tvm/ir</cite> folder contains the unified data structure and interfaces across for all IR function variants.
The components in <cite>tvm/ir</cite> are shared by <cite>tvm/relay</cite> and <cite>tvm/tir</cite>, notable ones include</p>
<ul class="simple">
<li><p>IRModule</p></li>
<li><p>类型</p></li>
<li><p>PassContext and Pass</p></li>
<li><p>Op</p></li>
</ul>
<p>Different variants of functions(e.g. relay.Function and tir.PrimFunc) can co-exist in an IRModule.
While these variants may not have the same content representation, they use the same data structure to represent types.
As a consequence, we use the same data structure to represent function (type) signatures of these variants.
The unified type system allows one function variant to call another function
once we clearly define the calling convention. This opens doors for future cross-function-variant optimizations.</p>
<p>We also provide a unified PassContext for configuring the pass behavior, and common composite passes to execute a pass pipeline.
The following code snippet gives an example of PassContext configuration.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># configure the behavior of the tir.UnrollLoop pass</span>
<span class="k">with</span> <span class="n">tvm</span><span class="o">.</span><span class="n">transform</span><span class="o">.</span><span class="n">PassContext</span><span class="p">(</span><span class="n">config</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;tir.UnrollLoop&quot;</span><span class="p">:</span> <span class="p">{</span> <span class="s2">&quot;auto_max_step&quot;</span><span class="p">:</span> <span class="mi">10</span> <span class="p">}}):</span>
    <span class="c1"># code affected by the pass context</span>
</pre></div>
</div>
<p>Op is the common class to represent all system-defined primitive operator/intrinsics.
Developers can register new Ops as well as their additional attributes(e.g. whether the Op is elementwise) to the system.</p>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="pass_infra.html">Pass Infrastructure</a></li>
</ul>
</div>
</div>
<div class="section" id="tvm-target">
<h2>tvm/target<a class="headerlink" href="#tvm-target" title="永久链接至标题">¶</a></h2>
<p>The target module contains all the code generators that translate an IRModule to a target runtime.Module.
It also provides a common <cite>Target</cite> class that describes the target.</p>
<p>The compilation pipeline can be customized according to the target by querying the attribute information
in the target and builtin information registered to each target id(cuda, opencl).</p>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="device_target_interactions.html">Device/Target Interactions</a></li>
</ul>
</div>
</div>
<div class="section" id="tvm-tir">
<h2>tvm/tir<a class="headerlink" href="#tvm-tir" title="永久链接至标题">¶</a></h2>
<p>TIR contains the definition of the low-level program representations. We use <cite>tir::PrimFunc</cite> to represent functions that can be transformed by TIR passes.
Besides the IR data structures, the tir module also defines a set of builtin intrinsics and their attributes via the common Op registry, as well as transformation passes in <cite>tir/transform</cite>.</p>
</div>
<div class="section" id="tvm-arith">
<h2>tvm/arith<a class="headerlink" href="#tvm-arith" title="永久链接至标题">¶</a></h2>
<p>This module is closely tied to the TIR. One of the key problems in the low-level code generation is the analysis of the indices’
arithmetic properties — the positiveness, variable bound, and the integer set that describes the iterator space. arith module provides
a collection of tools that do (primarily integer) analysis. A TIR pass can use these analyses to simplify and optimize the code.</p>
</div>
<div class="section" id="tvm-te">
<h2>tvm/te<a class="headerlink" href="#tvm-te" title="永久链接至标题">¶</a></h2>
<p>The name te stands for “tensor expression”. This is a domain-specific language module that allows us to construct <cite>tir::PrimFunc</cite> variants quickly by writing tensor expressions.
Importantly, a tensor expression itself is not a self-contained function that can be stored into IRModule. Instead, it is a fragment of IR that we can stitch together to build an IRModule.</p>
<p><cite>te/schedule</cite> provides a collection of scheduling primitives to control the function being generated. In the future, we might bring some of
these scheduling components to the a <cite>tir::PrimFunc</cite> itself.</p>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="inferbound.html">InferBound Pass</a></li>
<li class="toctree-l1"><a class="reference internal" href="hybrid_script.html">混合前端开发指南</a></li>
</ul>
</div>
</div>
<div class="section" id="tvm-topi">
<h2>tvm/topi<a class="headerlink" href="#tvm-topi" title="永久链接至标题">¶</a></h2>
<p>While possible to construct operators directly via TIR or tensor expressions (TE) for each use case it is tedious to do so.
<cite>topi</cite> (Tensor operator inventory) provides a set of pre-defined operators (in TE or TIR) defined by
numpy and found in common deep learning workloads. We also provide a collection of common schedule templates to obtain performant implementations across different target platforms.</p>
</div>
<div class="section" id="tvm-relay">
<h2>tvm/relay<a class="headerlink" href="#tvm-relay" title="永久链接至标题">¶</a></h2>
<p>Relay is the high-level functional IR used to represent full models. Various optimizations are defined in <cite>relay.transform</cite>. The Relay compiler defines multiple dialects,
and each dialect is designed to support specific styles of optimization. Notable ones include QNN(for importing pre-quantized models), VM(for lowering to dynamic virtual machine),
memory(for memory optimization).</p>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="relay_intro.html">Relay 介绍</a></li>
<li class="toctree-l1"><a class="reference internal" href="relay_op_strategy.html">Relay Operator Strategy</a></li>
<li class="toctree-l1"><a class="reference internal" href="convert_layout.html">Convert Layout Pass</a></li>
</ul>
</div>
</div>
<div class="section" id="tvm-autotvm">
<h2>tvm/autotvm<a class="headerlink" href="#tvm-autotvm" title="永久链接至标题">¶</a></h2>
<p>AutoTVM and AutoScheduler are both components which automate search based program optimization. This is rapidly evolving and primarily consists of:</p>
<ul class="simple">
<li><p>Cost models and feature extraction.</p></li>
<li><p>A record format for storing program benchmark results for cost model construction.</p></li>
<li><p>A set of search policies over program transformations.</p></li>
</ul>
<p>Automated program optimization is still an active research field. As a result, we have attempted to modularize the design so that researchers may quickly modify a
component or apply their own algorithms via the Python bindings, and
customize the search and plugin their algorithms from the Python binding.</p>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="benchmark.html">Benchmark Performance Log Format</a></li>
</ul>
</div>
</div>
<div class="section" id="frontends">
<h2>Frontends<a class="headerlink" href="#frontends" title="永久链接至标题">¶</a></h2>
<p>Frontends ingest models from different frameworks into the TVM stack.
<a class="reference internal" href="../reference/api/python/relay/frontend.html#module-tvm.relay.frontend" title="tvm.relay.frontend"><code class="xref py py-mod docutils literal notranslate"><span class="pre">tvm.relay.frontend</span></code></a> is the namespace for model ingestion APIs.</p>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="frontend/tensorflow.html">TensorFlow 前端</a></li>
</ul>
</div>
</div>
<div class="section" id="security">
<h2>安全<a class="headerlink" href="#security" title="永久链接至标题">¶</a></h2>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="security.html">Security Guide</a></li>
</ul>
</div>
</div>
<div class="section" id="microtvm">
<h2>microTVM<a class="headerlink" href="#microtvm" title="永久链接至标题">¶</a></h2>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="microtvm_design.html">microTVM设计文档</a></li>
<li class="toctree-l1"><a class="reference internal" href="model_library_format.html">Model Library Format</a></li>
</ul>
</div>
</div>
</div>


           </div>
           
          </div>
          

<footer>

    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="runtime.html" class="btn btn-neutral float-right" title="TVM runtime系统" accesskey="n" rel="next">下一个 <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="../dev/how_to/pytest_target_parametrization.html" class="btn btn-neutral float-left" title="Python Target Parametrization" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> 上一个</a>
      
    </div>

<div id="button" class="backtop"><img src="../_static//img/right.svg" alt="backtop"/> </div>
<section class="footerSec">
    <div class="footerHeader">
      <ul class="d-flex align-md-items-center justify-content-between flex-column flex-md-row">
        <li class="copywrite d-flex align-items-center">
          <h5 id="copy-right-info">© 2020 Apache Software Foundation | All right reserved</h5>
        </li>
      </ul>

    </div>

    <ul>
      <li class="footernote">Copyright © 2020 The Apache Software Foundation. Apache TVM, Apache, the Apache feather, and the Apache TVM project logo are either trademarks or registered trademarks of the Apache Software Foundation.</li>
    </ul>

</section>
</footer>
        </div>
      </div>

    </section>

  </div>
  

    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>

  </body>
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
    <!-- Theme Analytics -->
    <script>
    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
    })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

    ga('create', 'UA-75982049-2', 'auto');
    ga('send', 'pageview');
    </script>

    
   

</body>
</html>